In [1]:
import samplerate
from tensorflow import keras
from keras import losses
from keras.callbacks import EarlyStopping
from keras.optimizers import adam


import os
import sys
os.environ["CUDA_VISIBLE_DEVICES"]="4"
sys.path.insert(1,'/home/jgozlan/GIT/scripts/')
sys.path.insert(1,'/home/jgozlan/GIT/models/')
sys.path.insert(1, '/home/jgozlan/GIT/models_analysis/')
import tensorflow as tf
tf.test.is_gpu_available()

Using TensorFlow backend.
Out[1]:
True
In [2]:
import matplotlib.pyplot as plt
In [3]:
from imu_extractor import get_all_dataframes_for_forecasting, get_freq_magn, transform_dataset_into_freq_magn
from train_test_split import get_train_test_split
from baseline import find_frequency,repeat_period_forecast_acf, get_mean_line_forecast, get_flat_line_forecast
from metrics import compare_method, plot_few_prediction
from models import create_lstm_encoder_decoder
In [22]:
path_cluster0 = ['Run_Features']

path_cluster1 = ['CarDriving_Features']

path_cluster2 = ['Karting_Features',
                 'MotorcycleHelmet_Features',
                 'Scooter_Features','SkateboardChesty_Features',
                 'SnowboardSeeker_Features','Hiking_Features']

path_all_classes = path_cluster0 + path_cluster1 + path_cluster2

recording_freq = 6400.0
downsampling_freq = 200.0

all_dfs_classes = []

for i,c in enumerate(path_cluster0):
    dfs = get_all_dataframes_for_forecasting("Database/" + c, recording_freq, downsampling_freq)
    all_dfs_classes.append(dfs)
    
all_df = [item for sublist in all_dfs_classes for item in sublist]

GH010029.MP4.eis_dump.bin_features.json
GH010011.MP4.eis_dump.bin_features.json
GH019983.MP4.eis_dump.bin_features.json
GH010019.MP4.eis_dump.bin_features.json
GH010030.MP4.eis_dump.bin_features.json
GH010028.MP4.eis_dump.bin_features.json
GH019982.MP4.eis_dump.bin_features.json
GH010012.MP4.eis_dump.bin_features.json
GH019973.MP4.eis_dump.bin_features.json
In [23]:
lag = 500
ahead = 100
delay = 1
test_size = 0.2
dim = 12
target_index = [0,1,2]
classification = False

X_train, y_train, X_test, y_test = get_train_test_split(all_df, test_size, lag, ahead, delay, target_index, classification = False)
In [24]:
num_freq = 12

transformed_X_train = transform_dataset_into_freq_magn(X_train, downsampling_freq, num_freq)
transformed_X_test = transform_dataset_into_freq_magn(X_test, downsampling_freq, num_freq)

freq_dim_in = transformed_X_train.shape[1]
In [27]:
freq_dim_in
Out[27]:
288
In [129]:
import pickle
from numpy import unique
from numpy import where
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
import matplotlib.pyplot as plt
import pickle

cluster_classification_model = keras.models.load_model('models_analysis/cluster4_classification_model_new')

running_forecasting_model = keras.models.load_model('models_analysis/running_forecasting_multi_step1')

lda_model = pickle.load(open('models_analysis/lda_model.sav', 'rb'))
kmeans_model = pickle.load(open('models_analysis/kmeans_model.sav', 'rb'))
gmm_model = pickle.load(open('models_analysis/gmm_model.sav', 'rb'))
In [28]:
X_train.shape, X_test.shape, transformed_X_train.shape, transformed_X_test.shape
Out[28]:
((79012, 500, 12), (17050, 500, 12), (79012, 288), (17050, 288))
In [142]:
forecast = running_forecasting_model.predict(X_test)

forecast_x = forecast[:,:,0].reshape((forecast.shape[0],forecast.shape[1]))
forecast_y = forecast[:,:,1].reshape((forecast.shape[0],forecast.shape[1]))
forecast_z = forecast[:,:,2].reshape((forecast.shape[0],forecast.shape[1]))
In [143]:
import random
import numpy as np
from evaluation2 import plot_forecasts2, plot_metrics, get_metrics_for_forecast
import matplotlib.pyplot as plt
In [144]:
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))

mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))

mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))

mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))

mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))

mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))

cluster_probs = cluster_classification_model.predict(transformed_X_test)
cluster0_probs = cluster_probs[:,0]

mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
    
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z

mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z


for threshold in range(0,100,1):

    prob_threshold = threshold/100.0

    idx_lower = np.where(cluster0_probs < prob_threshold)[0]
    idx_higher = np.where(cluster0_probs > prob_threshold)[0]
    print(prob_threshold, len(idx_lower), len(idx_higher))
    
    if len(idx_lower) > 0:

        mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
        mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
        mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
        
        mae_threshold_x_l[threshold] = mae_low_x
        mae_threshold_y_l[threshold] = mae_low_y
        mae_threshold_z_l[threshold] = mae_low_z

        mse_threshold_x_l[threshold] = mse_low_x
        mse_threshold_y_l[threshold] = mse_low_y
        mse_threshold_z_l[threshold] = mse_low_z
    
    if len(idx_higher) > 0:
        mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
        mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
        mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
    
        mae_threshold_x_h[threshold] = mae_high_x
        mae_threshold_y_h[threshold] = mae_high_y
        mae_threshold_z_h[threshold] = mae_high_z

        mse_threshold_x_h[threshold] = mse_high_x
        mse_threshold_y_h[threshold] = mse_high_y
        mse_threshold_z_h[threshold] = mse_high_z

0.0 0 17050
0.01 3 17047
0.02 4 17046
0.03 5 17045
0.04 7 17043
0.05 7 17043
0.06 7 17043
0.07 7 17043
0.08 7 17043
0.09 8 17042
0.1 8 17042
0.11 8 17042
0.12 8 17042
0.13 8 17042
0.14 8 17042
0.15 8 17042
0.16 8 17042
0.17 8 17042
0.18 8 17042
0.19 8 17042
0.2 8 17042
0.21 8 17042
0.22 8 17042
0.23 8 17042
0.24 9 17041
0.25 9 17041
0.26 9 17041
0.27 9 17041
0.28 9 17041
0.29 9 17041
0.3 9 17041
0.31 9 17041
0.32 9 17041
0.33 9 17041
0.34 9 17041
0.35 9 17041
0.36 9 17041
0.37 9 17041
0.38 9 17041
0.39 10 17040
0.4 10 17040
0.41 10 17040
0.42 10 17040
0.43 10 17040
0.44 10 17040
0.45 10 17040
0.46 10 17040
0.47 10 17040
0.48 10 17040
0.49 10 17040
0.5 10 17040
0.51 10 17040
0.52 10 17040
0.53 10 17040
0.54 10 17040
0.55 10 17040
0.56 11 17039
0.57 11 17039
0.58 11 17039
0.59 11 17039
0.6 11 17039
0.61 11 17039
0.62 11 17039
0.63 11 17039
0.64 11 17039
0.65 11 17039
0.66 11 17039
0.67 11 17039
0.68 11 17039
0.69 11 17039
0.7 11 17039
0.71 11 17039
0.72 11 17039
0.73 11 17039
0.74 11 17039
0.75 11 17039
0.76 11 17039
0.77 11 17039
0.78 11 17039
0.79 11 17039
0.8 11 17039
0.81 11 17039
0.82 11 17039
0.83 11 17039
0.84 11 17039
0.85 11 17039
0.86 11 17039
0.87 12 17038
0.88 13 17037
0.89 13 17037
0.9 13 17037
0.91 13 17037
0.92 13 17037
0.93 13 17037
0.94 13 17037
0.95 14 17036
0.96 14 17036
0.97 14 17036
0.98 14 17036
0.99 18 17032
In [145]:
def plot_metrics_for_threshold(metrics_threshold, metrics_global):
    
    fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))

    for i, ax in enumerate(axs.flatten()):  
        ax.plot(range(ahead), metrics_global[i], label = "global")
        for threshold in range(90,100,1):
            ax.plot(range(ahead), metrics_threshold[i][threshold], label = threshold)
        ax.legend(loc="upper left")

    plt.show()
In [146]:
plot_metrics_for_threshold([mae_threshold_x_l,mae_threshold_y_l,mae_threshold_z_l], [mae_x_g, mae_y_g, mae_z_g])
In [147]:
plot_metrics_for_threshold([mse_threshold_x_l,mse_threshold_y_l,mse_threshold_z_l],[mse_x_g, mse_y_g, mse_z_g])
In [148]:
plot_metrics_for_threshold([mae_threshold_x_h,mae_threshold_y_h,mae_threshold_z_h], [mae_x_g, mae_y_g, mae_z_g])
In [149]:
plot_metrics_for_threshold([mse_threshold_x_h,mse_threshold_y_h,mse_threshold_z_h], [mse_x_g, mse_y_g, mse_z_g])
In [150]:
import numpy as np

batch_pipeline_size = 100
threshold  = 0.7

predictions_cluster = np.zeros((X_test.shape[0], ahead, 1 ))
predictions_global = np.zeros((X_test.shape[0], ahead, 1 ))
bad_indexes = np.zeros((X_test.shape[0]))


for i in range(0, len(X_test), batch_pipeline_size):
    
    if (i % 5000) == 0:
        print(i)

    batch_time = X_test[i: (i+ batch_pipeline_size)]
    batch_freq = transformed_X_test[i: (i+ batch_pipeline_size)]

    cluster_probs = cluster_classification_model.predict(batch_freq)
    cluster0_probs = cluster_probs[:,0]
    idx = np.where(cluster0_probs < threshold)[0]
    bad_indexes[i + idx] = 1


    
    

0
5000
10000
15000
In [152]:
import random
from evaluation2 import plot_forecasts2, plot_metrics
import matplotlib.pyplot as plt

def plot_examples_test(indexes, X_true, Y_true, forecast, k):
    
    random_indexes = random.sample(indexes, k)
    
    for i in random_indexes:
        
        fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
        
        x_input = X_true[i][:,0].reshape((lag,-1))
        y_input = X_true[i][:,1].reshape((lag,-1))
        z_input = X_true[i][:,2].reshape((lag,-1))

        x_true = Y_true[i][:,0].reshape((ahead,-1))
        y_true = Y_true[i][:,1].reshape((ahead,-1))
        z_true = Y_true[i][:,2].reshape((ahead,-1))

        x_forecast = forecast[i][:,0].reshape((ahead,-1))
        y_forecast = forecast[i][:,1].reshape((ahead,-1))
        z_forecast = forecast[i][:,2].reshape((ahead,-1))

        t = [len(x_input) + j for j in range(len(x_true))]
        #print(t)

        inputs = [x_input, y_input,z_input]
        outputs = [x_true , y_true , z_true]
        forecasts = [x_forecast, y_forecast, z_forecast]

        signal_labels = ['x_gyro signal', 'y_gyro_input signal','z_gyro signal']
        signal_colors = ['red','green','blue']

        forecast_labels = ['x_gyro forecast', 'y_gyro forecast','z_gyro forecast']
        forecast_colors = ['orange','lime','cyan']
        

        for i, ax in enumerate(axs.flatten()):
            ax.plot(range(len(inputs[i])), inputs[i], label = signal_labels[i], color = signal_colors[i])
            ax.plot(t, outputs[i], color = signal_colors[i])
            ax.plot(t, forecasts[i], label = forecast_labels[i], color = forecast_colors[i])
            ax.legend(loc="upper left")
        

        plt.show()
In [153]:
k = 10
plot_examples_test(list(np.where(bad_indexes==0)[0]), X_test, y_test, forecast, k)
In [154]:
k = 10
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
In [155]:
X_random_lda3 = lda_model.transform(transformed_X_test)
from sklearn.metrics.pairwise import euclidean_distances
In [156]:
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))

mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))

mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))

mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))

mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))

mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))

cluster_center = kmeans_model.cluster_centers_[-1]
distances_centroid_run = euclidean_distances(X_random_lda3, [cluster_center])

mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
    
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z

mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z


for threshold in range(0,100,1):

    current_threshold = threshold/10.0

    idx_lower = np.where(distances_centroid_run < current_threshold)[0]
    idx_higher = np.where(distances_centroid_run > current_threshold)[0]

    #print(current_threshold, len(idx_lower), len(idx_higher))
    
    if len(idx_lower) > 0:

        mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
        mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
        mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
        
        mae_threshold_x_l[threshold] = mae_low_x
        mae_threshold_y_l[threshold] = mae_low_y
        mae_threshold_z_l[threshold] = mae_low_z

        mse_threshold_x_l[threshold] = mse_low_x
        mse_threshold_y_l[threshold] = mse_low_y
        mse_threshold_z_l[threshold] = mse_low_z
    
    if len(idx_higher) > 0:
        mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
        mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
        mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
    
        mae_threshold_x_h[threshold] = mae_high_x
        mae_threshold_y_h[threshold] = mae_high_y
        mae_threshold_z_h[threshold] = mae_high_z

        mse_threshold_x_h[threshold] = mse_high_x
        mse_threshold_y_h[threshold] = mse_high_y
        mse_threshold_z_h[threshold] = mse_high_z

    print("THRESHOLD", current_threshold, len(idx_lower), len(idx_higher))

THRESHOLD 0.0 0 17050
THRESHOLD 0.1 10 17040
THRESHOLD 0.2 61 16989
THRESHOLD 0.3 210 16840
THRESHOLD 0.4 421 16629
THRESHOLD 0.5 789 16261
THRESHOLD 0.6 1208 15842
THRESHOLD 0.7 1807 15243
THRESHOLD 0.8 2444 14606
THRESHOLD 0.9 3060 13990
THRESHOLD 1.0 3726 13324
THRESHOLD 1.1 4410 12640
THRESHOLD 1.2 5164 11886
THRESHOLD 1.3 5967 11083
THRESHOLD 1.4 6666 10384
THRESHOLD 1.5 7378 9672
THRESHOLD 1.6 8025 9025
THRESHOLD 1.7 8669 8381
THRESHOLD 1.8 9282 7768
THRESHOLD 1.9 9852 7198
THRESHOLD 2.0 10345 6705
THRESHOLD 2.1 10833 6217
THRESHOLD 2.2 11255 5795
THRESHOLD 2.3 11669 5381
THRESHOLD 2.4 12026 5024
THRESHOLD 2.5 12405 4645
THRESHOLD 2.6 12728 4322
THRESHOLD 2.7 13037 4013
THRESHOLD 2.8 13291 3759
THRESHOLD 2.9 13548 3502
THRESHOLD 3.0 13728 3322
THRESHOLD 3.1 13933 3117
THRESHOLD 3.2 14109 2941
THRESHOLD 3.3 14271 2779
THRESHOLD 3.4 14426 2624
THRESHOLD 3.5 14577 2473
THRESHOLD 3.6 14745 2305
THRESHOLD 3.7 14913 2137
THRESHOLD 3.8 15068 1982
THRESHOLD 3.9 15208 1842
THRESHOLD 4.0 15369 1681
THRESHOLD 4.1 15480 1570
THRESHOLD 4.2 15586 1464
THRESHOLD 4.3 15746 1304
THRESHOLD 4.4 15884 1166
THRESHOLD 4.5 16014 1036
THRESHOLD 4.6 16130 920
THRESHOLD 4.7 16213 837
THRESHOLD 4.8 16285 765
THRESHOLD 4.9 16350 700
THRESHOLD 5.0 16393 657
THRESHOLD 5.1 16434 616
THRESHOLD 5.2 16461 589
THRESHOLD 5.3 16491 559
THRESHOLD 5.4 16533 517
THRESHOLD 5.5 16556 494
THRESHOLD 5.6 16588 462
THRESHOLD 5.7 16613 437
THRESHOLD 5.8 16653 397
THRESHOLD 5.9 16687 363
THRESHOLD 6.0 16713 337
THRESHOLD 6.1 16743 307
THRESHOLD 6.2 16761 289
THRESHOLD 6.3 16787 263
THRESHOLD 6.4 16813 237
THRESHOLD 6.5 16842 208
THRESHOLD 6.6 16867 183
THRESHOLD 6.7 16894 156
THRESHOLD 6.8 16916 134
THRESHOLD 6.9 16930 120
THRESHOLD 7.0 16945 105
THRESHOLD 7.1 16960 90
THRESHOLD 7.2 16978 72
THRESHOLD 7.3 16998 52
THRESHOLD 7.4 17008 42
THRESHOLD 7.5 17020 30
THRESHOLD 7.6 17026 24
THRESHOLD 7.7 17031 19
THRESHOLD 7.8 17036 14
THRESHOLD 7.9 17041 9
THRESHOLD 8.0 17044 6
THRESHOLD 8.1 17047 3
THRESHOLD 8.2 17049 1
THRESHOLD 8.3 17049 1
THRESHOLD 8.4 17050 0
THRESHOLD 8.5 17050 0
THRESHOLD 8.6 17050 0
THRESHOLD 8.7 17050 0
THRESHOLD 8.8 17050 0
THRESHOLD 8.9 17050 0
THRESHOLD 9.0 17050 0
THRESHOLD 9.1 17050 0
THRESHOLD 9.2 17050 0
THRESHOLD 9.3 17050 0
THRESHOLD 9.4 17050 0
THRESHOLD 9.5 17050 0
THRESHOLD 9.6 17050 0
THRESHOLD 9.7 17050 0
THRESHOLD 9.8 17050 0
THRESHOLD 9.9 17050 0
In [157]:
mae_x = [mae_threshold_x_l, mae_threshold_x_h]
mae_y = [mae_threshold_y_l, mae_threshold_y_h]
mae_z = [mae_threshold_z_l, mae_threshold_z_h]

fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))

for i, ax in enumerate(axs.flatten()): 
    if i == 0:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[0], axis = 1), label = "X- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[1], axis = 1), label = "X- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_x_g), label = "X- global")
    elif i == 1:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[0], axis = 1), label = "Y- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[1], axis = 1), label = "Y- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_y_g), label = "Y - global")
    else:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[0], axis = 1), label = "Z- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[1], axis = 1), label = "Z- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_z_g), label = "Z - global")

    ax.legend(loc="lower left")

plt.show()
In [158]:
def plot_metrics_for_euc_threshold(metrics_threshold, metrics_global):
    
    fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))

    for i, ax in enumerate(axs.flatten()):  
        ax.plot(range(ahead), metrics_global[i], label = "global", color='black')
        for threshold in range(5,80,5):
            ax.plot(range(ahead), metrics_threshold[i][threshold], label = threshold)
        ax.legend(loc="upper left")

    plt.show()
In [159]:
# LOWER THE BETTER
plot_metrics_for_euc_threshold([mae_threshold_x_l,mae_threshold_y_l,mae_threshold_z_l], [mae_x_g, mae_y_g, mae_z_g])
In [160]:
#HIGHER THE BETTER
plot_metrics_for_euc_threshold([mae_threshold_x_h,mae_threshold_y_h,mae_threshold_z_h], [mae_x_g, mae_y_g, mae_z_g])
In [161]:
threshold  = 4
bad_indexes = np.zeros((X_test.shape[0]))

idx_lower = np.where(distances_centroid_run < threshold)[0]
idx_higher = np.where(distances_centroid_run > threshold)[0]

bad_indexes[idx_higher] = 1
k = 20
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
In [162]:
threshold  = 7
bad_indexes = np.zeros((X_test.shape[0]))

idx_lower = np.where(distances_centroid_run < threshold)[0]
idx_higher = np.where(distances_centroid_run > threshold)[0]

bad_indexes[idx_higher] = 1

k = 20
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
In [ ]:
# GMMMMMMMMMMMMMMMM
In [163]:
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))

mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))

mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))

mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))

mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))

mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))

cluster_center = gmm_model.means_[0]
distances_centroid_run = euclidean_distances(X_random_lda3, [cluster_center])

mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
    
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z

mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z


for threshold in range(0,100,1):

    current_threshold = threshold/10.0

    idx_lower = np.where(distances_centroid_run < current_threshold)[0]
    idx_higher = np.where(distances_centroid_run > current_threshold)[0]

    #print(current_threshold, len(idx_lower), len(idx_higher))
    
    if len(idx_lower) > 0:

        mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
        mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
        mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
        
        mae_threshold_x_l[threshold] = mae_low_x
        mae_threshold_y_l[threshold] = mae_low_y
        mae_threshold_z_l[threshold] = mae_low_z

        mse_threshold_x_l[threshold] = mse_low_x
        mse_threshold_y_l[threshold] = mse_low_y
        mse_threshold_z_l[threshold] = mse_low_z
    
    if len(idx_higher) > 0:
        mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
        mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
        mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
    
        mae_threshold_x_h[threshold] = mae_high_x
        mae_threshold_y_h[threshold] = mae_high_y
        mae_threshold_z_h[threshold] = mae_high_z

        mse_threshold_x_h[threshold] = mse_high_x
        mse_threshold_y_h[threshold] = mse_high_y
        mse_threshold_z_h[threshold] = mse_high_z

    print("THRESHOLD", current_threshold, len(idx_lower), len(idx_higher))

THRESHOLD 0.0 0 17050
THRESHOLD 0.1 0 17050
THRESHOLD 0.2 0 17050
THRESHOLD 0.3 0 17050
THRESHOLD 0.4 0 17050
THRESHOLD 0.5 1 17049
THRESHOLD 0.6 3 17047
THRESHOLD 0.7 12 17038
THRESHOLD 0.8 27 17023
THRESHOLD 0.9 35 17015
THRESHOLD 1.0 55 16995
THRESHOLD 1.1 79 16971
THRESHOLD 1.2 113 16937
THRESHOLD 1.3 152 16898
THRESHOLD 1.4 189 16861
THRESHOLD 1.5 234 16816
THRESHOLD 1.6 264 16786
THRESHOLD 1.7 320 16730
THRESHOLD 1.8 395 16655
THRESHOLD 1.9 502 16548
THRESHOLD 2.0 622 16428
THRESHOLD 2.1 737 16313
THRESHOLD 2.2 850 16200
THRESHOLD 2.3 959 16091
THRESHOLD 2.4 1091 15959
THRESHOLD 2.5 1239 15811
THRESHOLD 2.6 1388 15662
THRESHOLD 2.7 1542 15508
THRESHOLD 2.8 1704 15346
THRESHOLD 2.9 1882 15168
THRESHOLD 3.0 2077 14973
THRESHOLD 3.1 2290 14760
THRESHOLD 3.2 2536 14514
THRESHOLD 3.3 2820 14230
THRESHOLD 3.4 3148 13902
THRESHOLD 3.5 3516 13534
THRESHOLD 3.6 3905 13145
THRESHOLD 3.7 4308 12742
THRESHOLD 3.8 4767 12283
THRESHOLD 3.9 5240 11810
THRESHOLD 4.0 5753 11297
THRESHOLD 4.1 6208 10842
THRESHOLD 4.2 6736 10314
THRESHOLD 4.3 7234 9816
THRESHOLD 4.4 7704 9346
THRESHOLD 4.5 8247 8803
THRESHOLD 4.6 8803 8247
THRESHOLD 4.7 9359 7691
THRESHOLD 4.8 9895 7155
THRESHOLD 4.9 10441 6609
THRESHOLD 5.0 10945 6105
THRESHOLD 5.1 11458 5592
THRESHOLD 5.2 11957 5093
THRESHOLD 5.3 12405 4645
THRESHOLD 5.4 12795 4255
THRESHOLD 5.5 13167 3883
THRESHOLD 5.6 13455 3595
THRESHOLD 5.7 13769 3281
THRESHOLD 5.8 14104 2946
THRESHOLD 5.9 14370 2680
THRESHOLD 6.0 14603 2447
THRESHOLD 6.1 14852 2198
THRESHOLD 6.2 15042 2008
THRESHOLD 6.3 15214 1836
THRESHOLD 6.4 15378 1672
THRESHOLD 6.5 15490 1560
THRESHOLD 6.6 15611 1439
THRESHOLD 6.7 15756 1294
THRESHOLD 6.8 15857 1193
THRESHOLD 6.9 15956 1094
THRESHOLD 7.0 16042 1008
THRESHOLD 7.1 16129 921
THRESHOLD 7.2 16228 822
THRESHOLD 7.3 16310 740
THRESHOLD 7.4 16400 650
THRESHOLD 7.5 16490 560
THRESHOLD 7.6 16580 470
THRESHOLD 7.7 16641 409
THRESHOLD 7.8 16733 317
THRESHOLD 7.9 16801 249
THRESHOLD 8.0 16876 174
THRESHOLD 8.1 16947 103
THRESHOLD 8.2 16987 63
THRESHOLD 8.3 17013 37
THRESHOLD 8.4 17028 22
THRESHOLD 8.5 17040 10
THRESHOLD 8.6 17045 5
THRESHOLD 8.7 17046 4
THRESHOLD 8.8 17049 1
THRESHOLD 8.9 17050 0
THRESHOLD 9.0 17050 0
THRESHOLD 9.1 17050 0
THRESHOLD 9.2 17050 0
THRESHOLD 9.3 17050 0
THRESHOLD 9.4 17050 0
THRESHOLD 9.5 17050 0
THRESHOLD 9.6 17050 0
THRESHOLD 9.7 17050 0
THRESHOLD 9.8 17050 0
THRESHOLD 9.9 17050 0
In [164]:
mae_x = [mae_threshold_x_l, mae_threshold_x_h]
mae_y = [mae_threshold_y_l, mae_threshold_y_h]
mae_z = [mae_threshold_z_l, mae_threshold_z_h]

fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))

for i, ax in enumerate(axs.flatten()): 
    if i == 0:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[0], axis = 1), label = "X- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[1], axis = 1), label = "X- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_x_g), label = "X- global")
    elif i == 1:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[0], axis = 1), label = "Y- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[1], axis = 1), label = "Y- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_y_g), label = "Y - global")
    else:
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[0], axis = 1), label = "Z- under threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[1], axis = 1), label = "Z- above threshold")
        ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_z_g), label = "Z - global")

    ax.legend(loc="lower left")

plt.show()